#!/usr/bin/perl

# $Id: crawler_manager.pl 247 2010-07-17 14:23:24Z dolbanavt $
# $Date: 2010-07-17 17:25:13 +0300 (Суб, 17 Июл 2010) $
# $HeadURL: https://robosearch.googlecode.com/svn/branches/reborn/crawler_manager.pl $
# $LastChangedRevision: 346 $

use lib::abs qw( base_lib lib );
use JIP::ToolSet;
use Carp qw( croak );
use Const::Fast qw( const );
use English qw( -no_match_vars );
use Pod::Usage qw( pod2usage );
use Getopt::Long 2.24 qw( :config no_auto_abbrev no_ignore_case );
use File::Spec::Functions qw( catfile );
use FindBin qw( $RealBin );
use File::Copy qw( move );
use IPC::Open3::Utils qw( put_cmd_in );
use JIP::Utils qw( get_log_name );
use Filesystem::Crawler;
use DBWrapperFactory;
use Log;
use Configurator;
use SQLWrapper;
use Proc::Lock;

our $VERSION = '1.0';

# more meaningfull names for pod2usage's -verbose parameter
const my $SHOW_USAGE_ONLY         => 0;
const my $SHOW_BRIEF_HELP_MESSAGE => 1;
const my $SHOW_FULL_MANUAL        => 2;

# more meaningfull names for pod2usage's -exitval parameter
const my $RETURN_SUCCESS     => 0;
const my $RETURN_FATAL_ERROR => 1;
const my $RETURN_BAD_OPTIONS => 2;

# configures behavior and functionality of put_cmd_in(). boolean arguments
const my $ARGS_FOR_OPEN3 => {
    close_stdin            => 1,
    carp_open3_errors      => 1,
    stop_read_on_open3_err => 1,
};

const my $CRAWLER_SUCCESS_MESSAGE => qr{--\scrawl\sloop\s--}xms;
const my $FILTER_SUCCESS_MESSAGE  => qr{exiting}xms;

#-- ui -------------------------------------------------------------------------
my $options_okay = GetOptions(
    'help'   => \my $is_help_requested,
    'conf=s' => \my $path_to_config_file,
);

# show appropriate help message
pod2usage( -exitval => $RETURN_BAD_OPTIONS, -verbose => $SHOW_USAGE_ONLY )
    unless $options_okay;

pod2usage( -exitval => $RETURN_SUCCESS, -verbose => $SHOW_FULL_MANUAL )
    if $is_help_requested;

# check required options
pod2usage(
    -exitval => $RETURN_BAD_OPTIONS,
    -message => 'required parameter "conf" not defined',
    -verbose => $SHOW_USAGE_ONLY,
) unless $path_to_config_file;

#-- init settings --------------------------------------------------------------
my $conf = Configurator->new( conf => $path_to_config_file )->get_settings;

#-- init log -------------------------------------------------------------------
our $log = Log->new(
    log_file  => catfile( $conf->crawler_manager->log_dir, get_log_name() ),
    log_level => $conf->crawler_manager->log_level,
);

#-- signal handlers ------------------------------------------------------------
require( catfile( $RealBin, 'signal_handlers.pl' ) );

#-- setup lock -----------------------------------------------------------------
$log->Note('setup lock');

my $process = Proc::Lock->new( lock_file => $conf->crawler_manager->lock_file );

$process->lock();

#-- db wrapper -----------------------------------------------------------------
my $db_wrapper = DBWrapperFactory->create_crawler_manager_wrapper(
    dbh => SQLWrapper->new(
        dsn  => $conf->db->dsn,
        user => $conf->db->user,
        pass => $conf->db->pass,
    )->get_db_handle,
);

#-- main -----------------------------------------------------------------------
$log->Note('run crawl loop...');

for my $each_host ( $db_wrapper->get_hosts_for_indexing( $conf->ageing ) ) {
    $log->Note(
        'for '. $each_host->get_ip_address .' with ID '. $each_host->get_id
    );

    my $crawler_fs = Filesystem::Crawler->new(
        host_object => $each_host,
        scratch_dir => $conf->crawler->scratch_dir,
        output_dir  => $conf->crawler->output_dir,
    );

    my $crawl_command = join q{ }, (
        'perl',
        catfile( $RealBin, 'ftp_crawler.pl' ),
        '--host='.        $each_host->get_ip_address,
        '--output_file='. $crawler_fs->scratch_file_location(),
        '--user='.        $conf->ftp->user,
        '--pass='.        $conf->ftp->pass,
        '--timeout='.     $conf->ftp->timeout,
    );

    $log->Debug( 'crawl command: '. $crawl_command );

    my $crawler_listing = run_command( $crawl_command );

    $log->Note('crawler finished with error') and next
        if not defined $crawler_listing;

    next if $crawler_listing !~ $CRAWLER_SUCCESS_MESSAGE;

    $log->Note('crawler finished with success message');

    $db_wrapper->update_status_for( $each_host->get_id );

    move(
        $crawler_fs->scratch_file_location(),
        $crawler_fs->output_file_location()
    ) or croak 'cannot move '. $crawler_fs->scratch_file_location() .' to '.
                               $crawler_fs->output_file_location()
                               .': '. $OS_ERROR;

    my $filter_fs = Filesystem::Filter->new(
        host_object => $each_host,
        scratch_dir => $conf->filter->scratch_dir,
        output_dir  => $conf->filter->output_dir,
    );

    my $filter_command = join q{ }, (
        'perl',
        catfile( $RealBin, 'filter.pl' ),
        '--input_file='.  $crawler_fs->output_file_location(),
        '--output_file='. $filter_fs->scratch_file_location(),
        '--host_id='.     $each_host->get_id,
    );

    $filter_command .= q{ } . '--memoize'
        if defined $conf->filter->memoize_requested;

    $log->Debug( 'filter command: '. $filter_command );

    my $filter_listing = run_command( $filter_command );

    $log->Note('filter finished with error') and next
        if not defined $filter_listing;

    next if $filter_listing !~ $FILTER_SUCCESS_MESSAGE;

    $log->Note('filter finished with success message');

    move(
        $filter_fs->scratch_file_location(),
        $filter_fs->output_file_location()
    ) or croak 'cannot move '. $filter_fs->scratch_file_location() .' to '.
                               $filter_fs->output_file_location()
                               .': '. $OS_ERROR;
}

#-- finalyze -------------------------------------------------------------------
$log->Note('-- finalyze -----------------------------------------------------');
$log->Note('unset lock...') and $process->unlock() and $log->Note('exiting');

#-- subroutines ----------------------------------------------------------------
sub run_command {
    my $command = shift;

    put_cmd_in( $command, \my $stdout, \my $stderr, $ARGS_FOR_OPEN3 );

    if ( $stderr ) {
        chomp $stderr;
        croak 'ошибка выполнения команды `'. $command .'`: '. $stderr;
    }

    return $stdout;
}

#-- end ------------------------------------------------------------------------

__END__

=head1 NAME

crawler_manager.pl - менеджер краулеров

=head1 SYNOPSIS

crawler_manager.pl [options]

 Options:
   --help            show full help message
   --conf            path to config file

=head1 USAGE

/path/to/crawler_manager.pl --help

/path/to/crawler_manager.pl --conf=./robo_conf.pm

=head1 DESCRIPTION

Менеджер процессов индексации серверов и фильтрации полученного
контента.

=head1 DEPENDENCIES

lib::abs (from CPAN)

Const::Fast (from CPAN)

IPC::Open3::Utils (from CPAN)

JIP::ToolSet (from base_lib)

Log (from base_lib)

Configurator (from base_lib)

SQLWrapper (from base_lib)

Proc::Lock (from base_lib)

=head1 EXIT STATUS

0 - программа выполнена успешно

1 - критическая ошибка программы

2 - неверные параметры командной строки

=head1 AUTHOR

Vladimir Zhavoronkov, <dolbanavt (at) Google Mail>

http://dolbanavt.livejournal.com/

=head1 LICENSE AND COPYRIGHT

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

=cut
